{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Pecu https://github.com/pecu/PyTorch_CSX/blob/master/01_PyTorch_Basic/01_TensorDef.ipynb\n",
    "* 作者: 莫烦 https://morvanzhou.github.io/tutorials/machine-learning/torch/2-02-variable/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fe2934dda50>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensor  &  numpy array\n",
    "* PyTorch tensor 本質上和 numpy array差不多，可以互相轉換。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3 2 1]\n",
      " [4 5 6]]\n",
      "\n",
      " 3  2  1\n",
      " 4  5  6\n",
      "[torch.FloatTensor of size 2x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "list1 = [[3,2,1], [4,5,6]]\n",
    "\n",
    "# list > numpy:\n",
    "arr1 = np.array(list1)\n",
    "print(arr1)\n",
    "\n",
    "\n",
    "# list > pytorch-tensor\n",
    "tr1 = torch.FloatTensor(list1)\n",
    "print(tr1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 3  2  1\n",
      " 4  5  6\n",
      "[torch.LongTensor of size 2x3]\n",
      "\n",
      "[[ 3.  2.  1.]\n",
      " [ 4.  5.  6.]]\n"
     ]
    }
   ],
   "source": [
    "# numpy => pytorch tensor\n",
    "tr2 = torch.from_numpy(arr1) # dtype same as numpy\n",
    "print(tr2)\n",
    "\n",
    "\n",
    "# pytorch tensor => numpy\n",
    "arr2 = tr1.numpy()\n",
    "print(arr2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pytorch Variable\n",
    "* 在Torch中的 Variable 就是一個存放可變化數值的位置，裡面值可不停的變化，如果用一個變量進行計算，那結果也是一個同類型的變量。\n",
    "* 變量計算時，會默默搭建計算圖將所有的計算步驟連接起來，最後進行誤差反向傳遞的時候，一次性將所有變量裡面的微分值都計算出來。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 1  2\n",
       " 3  4\n",
       "[torch.FloatTensor of size 2x2]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 先產生數值\n",
    "tensor = torch.FloatTensor([[1,2],[3,4]])\n",
    "\n",
    "\n",
    "# 把值放到變數籃子裡 ( requires_grad表示是否參與誤差反向傳播 & 要不要計算梯度 )\n",
    "variable = Variable(tensor, requires_grad=True)\n",
    "variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 7.5000\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 變數做計算\n",
    "v_out = torch.mean(variable * variable)   \n",
    "print(v_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 0.5000  1.0000\n",
      " 1.5000  2.0000\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "v_out.backward() \n",
    "print(variable.grad)    # 初始 Variable 的梯度\n",
    "# v_out = (1/4) * sum(variable * variable)  =>  v_out 對 variable 微分 ＝ 1/2 (variable) #次方下乘,次方減 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1  2\n",
      " 3  4\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 變數很多運算做不了，可以用 variable.data 取出數值（變成 Tensor）\n",
    "print(variable.data)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Review: Embedding\n",
    "* 將一組數字轉成一組向量，且數字與向量嚴格一對一對應。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 0.6614  0.2669  0.0617  0.6213 -0.4519 -0.1661 -1.5228  0.3817 -1.0276 -0.5631\n",
      "[torch.FloatTensor of size 1x10]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 有一組字串\n",
    "'I love machine learning !'\n",
    "\n",
    "# 字典化(我們只能處理數字)\n",
    "word_to_idx = {'I': 0, 'love': 1, 'machine':2, 'learning':3, '!':4}  \n",
    "\n",
    "# 張量\n",
    "word_tensor = torch.LongTensor([word_to_idx[\"I\"]])\n",
    "\n",
    "# 裝進 Variable\n",
    "word_var = Variable(word_tensor)\n",
    "\n",
    "# 把這個 Variable 變成一串數字 \n",
    "embeds = nn.Embedding(1, 10)\n",
    "word_embed = embeds(word_var)\n",
    "\n",
    "print(word_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 0.3711 -0.1741  1.1385 -1.4325  0.2623  1.2968  1.0816  0.4075 -0.9404 -1.3203\n",
      " 1.6002  1.3146 -0.6118 -0.9419 -0.1675 -1.6990 -2.0724  1.5600 -0.5075 -1.6533\n",
      "-0.0907 -1.0677 -0.4728 -0.0388 -0.0063 -0.1100  0.1423  0.2453 -0.6245 -0.7920\n",
      " 1.2385  0.8845 -0.4387 -0.9479 -1.0857 -0.0113  0.5639 -0.5461 -0.5868 -0.1270\n",
      " 1.2385  0.8845 -0.4387 -0.9479 -1.0857 -0.0113  0.5639 -0.5461 -0.5868 -0.1270\n",
      "[torch.FloatTensor of size 5x10]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 一次將一句話向量化(裝在變數裡)\n",
    "word_tensor = torch.LongTensor([ word_to_idx[\"I\"],\n",
    "                                 word_to_idx[\"love\"],\n",
    "                                 word_to_idx[\"machine\"],\n",
    "                                 word_to_idx[\"learning\"],\n",
    "                                \n",
    "                                 word_to_idx[\"learning\"] # 重複\n",
    "                               ])\n",
    "\n",
    "# 裝進 Variable\n",
    "word_var = Variable(word_tensor)\n",
    "\n",
    "# 把這個 Variable 變成一串數字 \n",
    "embeds = nn.Embedding(5, 10)\n",
    "word_embed = embeds(word_var)\n",
    "\n",
    "print(word_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 0.7491 -1.5348  0.3031  0.4657 -1.7558 -0.2372  0.8212 -0.9065  0.3359  1.4863\n",
      " 0.1520 -1.3443  0.6541  2.2225  0.2947 -0.8987  0.6914 -0.3367 -0.9926  0.4029\n",
      " 0.4275 -1.1540 -0.1806  1.4955  0.7525 -0.3977 -1.2727 -0.1921  0.6353 -1.3944\n",
      "-0.9095  0.1111 -0.0352 -0.8154  0.2774  0.6695  0.9860  0.1008  0.7559 -0.8964\n",
      " 0.7890  0.6291 -0.5083 -0.3679 -0.9695 -0.3280 -0.6444 -0.1242  0.1136  1.0938\n",
      "[torch.FloatTensor of size 5x10]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 常犯錯誤：\n",
    "\n",
    "# 把一個 index 改成 6 則超出原本 nn.Embedding(5, 10) [0,1,2,3,4] 的範圍\n",
    "word_to_ix = {'I': 0, 'love': 1, 'machine':2, 'learning':3, '!':4}  \n",
    "\n",
    "# 一次將一句話向量化(裝在變數裡)\n",
    "word_tensor = torch.LongTensor([ word_to_ix[\"I\"],\n",
    "                                 word_to_ix[\"love\"],\n",
    "                                 word_to_ix[\"machine\"],\n",
    "                                 word_to_ix[\"learning\"],\n",
    "                                 word_to_ix[\"!\"]\n",
    "                               ])\n",
    "\n",
    "\n",
    "\n",
    "word_var = Variable(word_tensor)\n",
    "# 把這個 Variable 變成一串數字 \n",
    "embeds = nn.Embedding(5, 10)\n",
    "word_embed = embeds(word_var)\n",
    "\n",
    "print(word_embed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 順便名詞解釋：\n",
    "##### 詞向量\n",
    "詞向量(word vector）是近年在自然語言處理中被廣泛使用的一種技術。一言以蔽之，我們想使用一個向量來表示每一個詞，如此一來，就能把一段由許多詞組成的文句，轉換成一個個詞向量來表示，並把這樣數值化的資料，送到模型裡做後續的應用。\n",
    "\n",
    "###### one-hot representation 直接表示法：\n",
    "最直觀的詞表示方法, 把每個詞表示為一個很長的向量，維度是詞表大小，其中大多數元素為0，只有一個維度的值為1代表了當前的詞。<br>\n",
    "本方式無法表格每個詞的遠近關係。<br>\n",
    "ex: 上面的 'I' =    [1, 0, 0, 0, 0]\n",
    "ex: 上面的 'love' = [0, 1, 0, 0, 0]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "###### word embedding 結合神經網絡訓練：\n",
    "透詞直接向量化後去配合模型訓練後，讓模型自己找出最好的向量表示方式。<br>\n",
    "可以解決one-hot 維度過大的缺點，將原來稀疏的巨大維度壓縮嵌入到一個更小維度的空間。<br>\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "###### word2vec 語言模型：\n",
    "Word2Vec是從大量文本語料中學習語義知識的一種模型，目的想衣文本學到更好的向量化表示。<br>\n",
    "Word2Vec = Continuous Bag Of Words（CBOW）+ k-skip-N-gram <br>\n",
    "* k-skip-N-gram: 輸入目標詞詞向量，預測出可能的前後詞<br> \n",
    "* CBOW: 輸入前後詞(contex word)詞向量，要預測出可能的目標詞(target word)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pytorch 兩個函數說明(Attention 會用到)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 3.3969\n",
      "-3.5469\n",
      " 0.3958\n",
      "-0.0856\n",
      " 0.3659\n",
      "-1.8985\n",
      " 0.3397\n",
      " 0.1151\n",
      " 1.2896\n",
      "-4.5139\n",
      "[torch.FloatTensor of size 10]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Applies a linear transformation to the incoming data: y = Ax + b\n",
    "m = nn.Linear(5, 10, bias=True)\n",
    "\n",
    "\n",
    "input = Variable(torch.FloatTensor([1,2,3,4,5]))\n",
    "output = m(input)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 3, 5])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch.bmm(很多矩陣相乘)\n",
    "batch1 = torch.randn(10, 3, 4) # 10 個 3X4\n",
    "batch2 = torch.randn(10, 4, 5) # 10 個 4X5\n",
    "\n",
    "\n",
    "res = torch.bmm(batch1, batch2)\n",
    "res.size()# 10 個 3X5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
